package univer.oko.queryAnalizer.sqlEditor;

import java.util.ArrayList;
import java.util.HashSet;

/**
 * Created by Sergey Kharlamov.
 * Date: 22.07.2006
 * Time: 16:56:10
 */
public class TextMiniTokenizer {
    /**
     * Klass vypolnjaet primitivnyj razbor simvolov teksta
     */
    private class MiniStream {
        public MiniStream(String _sourceString, int _ptr) {
            sourceString = _sourceString;
            curPtr = _ptr;
        }

        private String sourceString;
        private int curPtr;

        // vozvraschaet true esli kursorom dostignut konets massiva simvolov
        public boolean end() {
            return curPtr >= sourceString.length();
        }

        // vozvraschaet tekuschuju pozitsiju v massive simvolov
        public int prt() {
            return curPtr;
        }

        //  ustanovka tekuschego ukazatelja
        public void setprt(int _newPtr) {
            curPtr = _newPtr;
        }

        // perekhod k sledujuschemu simvolu
        public void next() {
            if (curPtr < sourceString.length()) curPtr++;
        }

        // podstroka s.. po..
        public String substring(int _beginIndex, int _endIndex) {
            if (_endIndex < sourceString.length()) _endIndex++;
            return sourceString.substring(_beginIndex, _endIndex);
        }

        // vozvraschaet simvol po smescheniju ot tekuschej pozitsii kursora
        public char get(int _offset) {
            return ((curPtr + _offset) >= sourceString.length()) ? 0 : sourceString.charAt(curPtr + _offset);
        }

        // vozvraschaet simvol po tekuschej pozitsii kursora
        public char get() {
            return get(0);
        }

        // sravnivaet simvol po smescheniju ot
        // tekuschej pozitsii kursora s zadannym simvolom
        public boolean isEquals(char _with, int _offset) {
            return get(_offset) == _with;
        }

        // sravnivaet tekuschij simvol s zadannym
        public boolean isEquals(char _with) {
            return isEquals(_with, 0);
        }

        // proverka na bukvu
        public boolean isCharacter() {
            return Character.isLetter(get());
        }

        // proverka na tsifru - nachalo dolzhno byt' objazatel'no tsifroj
        public boolean isDigit() {
            return Character.isDigit(get());
        }

        // proverka na tsifru
        public boolean isDigitBody() {
            return Character.isDigit(get())
                    || (get() == '.')
                    || (get() == 'e')
                    || (get() == 'E')
                    || (get() == 'x')
                    || (get() == 'X');
        }

        // proverka na spetsial'nyj simvol
        public boolean isSpecial() {
            return (get() == '#') || (get() == '$') || (get() == '_') || (get() == '@');
        }

        //  nachalo konstanty daty
        public boolean isDateConstantStart() {
            return (get() == '#');
        }

        // proverka na simvol, prinadlezhaschij spetsial'nomu naboru
        public boolean isSpecialTextSymbol() {
            return
                    (get() == '=')
                            || (get() == '!')
                            || (get() == '?')

                            || (get() == '*')
                            || (get() == '/')
                            || (get() == '-')
                            || (get() == '+')

                            || (get() == '|')
                            || (get() == '&')

                            || (get() == ';')
                            || (get() == ':')
                            || (get() == '.')
                            || (get() == ',')

                            || (get() == '>')
                            || (get() == '<')

                            || (get() == '(')
                            || (get() == ')')
                            || (get() == '{')
                            || (get() == '}')
                            || (get() == '[')
                            || (get() == ']')

                            || (get() == '%')
                            || (get() == '@')
                            || (get() == '#')
                            || (get() == '^')
                            || (get() == '\\')
                            || (get() == '\'')
                            || (get() == '"');
        }

        // proverka na simvol, s kotorogo mozhet nachinat'sja identifikator
        public boolean isStartIdentifierChar() {
            return isCharacter() || isSpecial();
        }

        // proverka na simvol, kotoryj mozhet vkhodit' v identifikator
        public boolean isIdentifierChar() {
            return isCharacter() || isDigit() || isSpecial();
        }

        // proverka na nachalo slozhnogo identifikatora ""
        public boolean isComplexIdentifierChar() {
            return isEquals('"');
        }

        // proverka na nachalo slozhnogo identifikatora []
        public boolean isComplex2IdentifierStartChar() {
            return isEquals('[');
        }

        // proverka na konets slozhnogo identifikatora []
        public boolean isComplex2IdentifierStopChar() {
            return isEquals(']');
        }

        // proverka na ;
        public boolean isSemi() {
            return isEquals(';');
        }

        // proverka na nachalo strokovoj konstanty
        public boolean isStringConstantChar() {
            return isEquals('\'');
        }

        //proverka na odnostrochnyj kommentarij --
        public boolean isSimpleCommentStart() {
            return isEquals('-') && isEquals('-', 1);
        }

        //proverka na konets odnostrochnogo kommentarija (konets stroki)
        public boolean isSimpleCommentEnd() {
            return isEquals('\r') || isEquals('\n');
        }

        //proverka na nachalo kommentarija /*
        public boolean isComplexCommentStart() {
            return isEquals('/') && isEquals('*', 1);
        }

        //proverka na konets kommentarija */
        public boolean isComplexCommentEnd() {
            return isEquals('*') && isEquals('/', 1);
        }

        public boolean isWhiteSpace() {
            return Character.isWhitespace(get());
        }
    }

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    //  spisok vozmozhnykh sostojanij
    /**
     * vne vsekh sostojanij
     */
    private final static int STATE_DEFAULT = 0;
    /**
     * proiskhodit razbor odnostrochnogo kommentarja
     */
    private final static int STATE_SIMPLE_COMMENT = 1;
    /**
     * proiskhodit razbor slozhnogo kommentarja
     */
    private final static int STATE_COMPLEX_COMMENT = 2;
    /**
     * proiskhodit razbor prostogo identifikatora
     */
    private final static int STATE_SIMPLE_IDENTIFIER = 3;
    /**
     * proiskhodit razbor slozhnogo identifikatora ""
     */
    private final static int STATE_COMPLEX_IDENTIFIER = 4;
    /**
     * proiskhodit razbor strokovoj konstanty
     */
    private final static int STATE_STRING_CONST = 5;
    /**
     * proiskhodit razbor chislovoj konstany
     */
    private final static int STATE_NUMBER_CONSTANT = 6;
    /**
     * proiskhodit razbor spetsial'nogo teksta
     */
    private final static int STATE_SPEC_TEXT = 7;
    /**
     * proiskhodit razbor slozhnogo identifikatora []
     */
    private final static int STATE_COMPLEX_IDENTIFIER_2 = 8;
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

//-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    /**
     * pri obrabotke TOKEN'a khranit ukazatel' na ego pervyj simvol
     */
    private int m_startTokenIndex;

    /**
     * poluchit' indeks pervogo simvola tol'ko chto vozvraschennogo TOKEN'a
     */
    private int getLastTokenFirstCharIndex() {
        return m_startTokenIndex;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    /**
     * pri obrabotke TOKEN'a khranit ukazatel' na ego poslednij simvol
     */
    private int m_finishTokenIndex;

    /**
     * poluchit' indeks poslednego simvola tol'ko chto vozvraschennogo TOKEN'a
     */
    private int getLastTokenLastCharIndex() {
        return m_finishTokenIndex;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    /**
     * tip obrabotannogo tokena - slozhnyj identifikator
     */
    private static final int TT_COMPLEX_IDENTIFIER = 0;
    /**
     * tip obrabotannogo tokena - identifikator
     */
    private static final int TT_IDENTIFIER = 2;
    /**
     * tip obrabotannogo tokena - znak =, ==, (, ), >=, * ...
     */
    private static final int TT_SPEC_SIGN = 3;
    /**
     * tip obrabotannogo tokena - strokovaja konstanta
     */
    private static final int TT_STRING = 4;
    /**
     * tip obrabotannogo tokena - chislovaja konstanta
     */
    private static final int TT_NUMBER = 5;
    /**
     * tip obrabotannogo tokena - kommentarij
     */
    private static final int TT_COMMENT = 6;

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    /**
     * tip poslednego TOKEN'a
     */
    private int m_dTokenType;

    /**
     * tip tol'ko chto vozvraschennogo TOKEN'a
     */
    private int getLastTokenType() {
        return m_dTokenType;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    private class CategoryDescription {
        boolean bStringConstant;
        boolean bNumberConstant;
        boolean bComment;
        boolean bAnySpecSign;
        boolean bAnyIdentifier;
        HashSet aIdentifiers = new HashSet();

        public CategoryDescription(boolean pStringConstant,
                                   boolean pNumberConstant, boolean pComment, boolean pAnySpecSign, boolean pAnyIdentifier, String[] pIdentifiers) {
            bStringConstant = pStringConstant;
            bNumberConstant = pNumberConstant;
            bComment = pComment;
            bAnySpecSign = pAnySpecSign;
            bAnyIdentifier = pAnyIdentifier;
            {  //  zapolnit' kartu

                for (int i = 0; i < pIdentifiers.length; i++)
                    aIdentifiers.add(pIdentifiers[i].toUpperCase());
            }

        }
    }

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    private ArrayList m_dCategories = new ArrayList();
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * dobavljaet kategoriju.
     *
     * @return nomer sozdannoj kategorii
     */
    public int addTokenCategory(boolean pStringConstant, boolean pNumberConstant,
                                boolean pComment, boolean pAnySpecSign, boolean pAnyIdentifier, String[] pIdentifiers) {
        CategoryDescription _newCat = new CategoryDescription(pStringConstant, pNumberConstant,
                pComment, pAnySpecSign, pAnyIdentifier, pIdentifiers);
        m_dCategories.add(_newCat);
        return m_dCategories.size() - 1;
    }

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    private boolean isLastTokenInThisCategory(String _lastToken, int _categoryNo) {
        CategoryDescription _cat = (CategoryDescription) m_dCategories.get(_categoryNo);
        int tt = getLastTokenType();
        //  spetsial'nye sluchai
        if ((tt == TT_COMMENT) && _cat.bComment) return true;
        if ((tt == TT_STRING) && _cat.bStringConstant) return true;
        if (((tt == TT_IDENTIFIER) || (tt == TT_COMPLEX_IDENTIFIER)) && _cat.bAnyIdentifier) return true;
        if ((tt == TT_SPEC_SIGN) && _cat.bAnySpecSign) return true;
        if ((tt == TT_NUMBER) && _cat.bNumberConstant) return true;
        //  konkretnye tokeny (TOL'KO sredi prostykh identifikatorov!)
        if ((_cat.aIdentifiers != null) && (tt == TT_IDENTIFIER))
            if (_cat.aIdentifiers.contains(_lastToken))
                return true;
        //  vsjo
        return false;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * poisk kategorii, kotoroj prinadlezhit dannyj token. -1, esli ne byla najdena
     */
    private int findCategoryOfLastToken(String _lastToken) {
        for (int i = 0; i < m_dCategories.size(); i++)
            if (isLastTokenInThisCategory(_lastToken, i))
                return i;
        return -1;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * Poluchit' sledujuschij TOKEN.
     * Vozvraschaet null, esli potok zakonchilsja.
     */
    private String getNextToken(MiniStream _stream) {
        while (true) {
            //  obrabotat' simvol
            switch (m_state) {
                //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_DEFAULT:
                    //  vykhod
                    if (_stream.end())
                        return null;
                        //  proiskhodit poisk sledujuschego token'a
                    else if (_stream.isStartIdentifierChar()) { //  nachalo obychnogo identifikatora
                        //  novoe sostojanie
                        m_state = STATE_SIMPLE_IDENTIFIER;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    } else if (_stream.isComplexIdentifierChar()) { //  nachalo slozhnogo identifikatora ""
                        //  novoe sostojanie
                        m_state = STATE_COMPLEX_IDENTIFIER;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    } else if (_stream.isComplex2IdentifierStartChar()) { //  nachalo slozhnogo identifikatora []
                        //  novoe sostojanie
                        m_state = STATE_COMPLEX_IDENTIFIER_2;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    } else if (_stream.isStringConstantChar()) { //  nachalo strokovoj konstanty
                        //  novoe sostojanie
                        m_state = STATE_STRING_CONST;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    } else if (_stream.isSimpleCommentStart()) { //  nachalo prostogo kommentarija
                        //  novoe sostojanie
                        m_state = STATE_SIMPLE_COMMENT;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                        //  propustit' eschjo odin simvol
                        _stream.next();
                    } else if (_stream.isComplexCommentStart()) { //  nachalo slozhnogo kommentarija
                        //  novoe sostojanie
                        m_state = STATE_COMPLEX_COMMENT;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                        //  propustit' eschjo odin simvol
                        _stream.next();
                    } else if (_stream.isDigit()) { //  nachalo chislovoj konstanty
                        //  novoe sostojanie
                        m_state = STATE_NUMBER_CONSTANT;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    }
                    //  roverka na spetsial'nyj token ""
                    else if (_stream.isSpecialTextSymbol()) { //  nachalo spetsial'nogo tokena
                        //  novoe sostojanie
                        m_state = STATE_SPEC_TEXT;
                        //  nachalo identifikatora
                        m_startTokenIndex = _stream.prt();
                    }
                    //  v ljubom sluchae - sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_SPEC_TEXT:
                    //  v ljubom sluchae kazhdyj spetsial'nyj simvol obrabatyvat' kak otdel'nyj token.
                    //if (_stream.end() || !_stream.isSpecialTextSymbol())
                {
                    m_state = STATE_DEFAULT; //  vsjo, kommentarij zakonchilsja
                    //  tip tokena
                    m_dTokenType = TT_SPEC_SIGN;
                    //  sledujuschij simvol
                    m_finishTokenIndex = _stream.prt() - 1;
                    return _stream.substring(m_startTokenIndex, m_finishTokenIndex);
                }
                // !!! sjuda dokhodit' ne dolzhno
                //   inache - sledujuschij simvol
                // _stream.next();
                //break;
                //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_NUMBER_CONSTANT:
                    if (_stream.end() || !_stream.isDigitBody()) {
                        m_state = STATE_DEFAULT; //  vsjo, konstanta zakonchilas'
                        //  tip tokena
                        m_dTokenType = TT_NUMBER;
                        //  sledujuschij simvol
                        m_finishTokenIndex = _stream.prt() - 1;
                        return _stream.substring(m_startTokenIndex, m_finishTokenIndex);
                    }
                    //  inache - sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_SIMPLE_IDENTIFIER:
                    if (_stream.end() || !_stream.isIdentifierChar()) {
                        //  vsjo, identifikator zakonchilsja
                        m_state = STATE_DEFAULT;
                        //  tip tokena
                        m_dTokenType = TT_IDENTIFIER;
                        //  prigotovit' otvet, ne perekhodit' na sledujuschij simvol
                        m_finishTokenIndex = _stream.prt() - 1;
                        //  vernut' v upper case
                        return _stream.substring(m_startTokenIndex, m_finishTokenIndex).toUpperCase();
                    }
                    //  inache - sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_COMPLEX_IDENTIFIER:
                    if (_stream.end() || _stream.isComplexIdentifierChar()) {
                        //  sledujuschij simvol
                        _stream.next();

                        //  proverit' sledujuschij simvol - ne dvojnoj li eto ""
                        if (_stream.end() || !_stream.isComplexIdentifierChar()) { //  dejstvitel'noe okonchanie
                            //  vsjo, identifikator zakonchilsja
                            m_state = STATE_DEFAULT;
                            //  tip tokena
                            m_dTokenType = TT_COMPLEX_IDENTIFIER;
                            //  prigotovit' otvet
                            m_finishTokenIndex = _stream.prt() - 1;
                            //  vernut' NE raskodirovannym
                            return /*decode(*/_stream.substring(m_startTokenIndex, m_finishTokenIndex)/*)*/;
                        }
                        //  inache - pristo propustit' oba
                    }
                    //  inache - sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_COMPLEX_IDENTIFIER_2:
                    if (_stream.end() || _stream.isComplex2IdentifierStopChar()) {
                        //  sledujuschij simvol
                        _stream.next();

                        //  proverit' sledujuschij simvol - ne dvojnoj li eto ]]
                        if (_stream.end() || !_stream.isComplex2IdentifierStopChar()) { //  dejstvitel'noe okonchanie
                            //  vsjo, identifikator zakonchilsja
                            m_state = STATE_DEFAULT;
                            //  tip tokena
                            m_dTokenType = TT_COMPLEX_IDENTIFIER;
                            //  prigotovit' otvet
                            m_finishTokenIndex = _stream.prt() - 1;
                            //  vernut' NE raskodirovannym
                            return /*decode(*/_stream.substring(m_startTokenIndex, m_finishTokenIndex)/*)*/;
                        }
                        //  inache - pristo propustit' oba
                    }
                    //  inache - sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_STRING_CONST:
                    if (_stream.end() || _stream.isStringConstantChar()) {
                        //  sledujuschij simvol
                        _stream.next();

                        //  proverit' sledujuschij simvol - ne dvojnoj li eto ''
                        if (_stream.end() || !_stream.isStringConstantChar()) {
                            //  dejstvitel'noe okonchanie konstanty
                            m_state = STATE_DEFAULT;
                            //  tip tokena
                            m_dTokenType = TT_STRING;
                            //  prigotovit' otvet
                            m_finishTokenIndex = _stream.prt() - 1;
                            //  vernut'
                            return _stream.substring(m_startTokenIndex, m_finishTokenIndex);
                        } else
                            //  inache - pristo propustit' oba simvola ''
                            _stream.next();
                    }
                    //  inache - sledujuschij simvol
                    else
                        _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_SIMPLE_COMMENT:
                    if (_stream.end() || _stream.isSimpleCommentEnd()) {
                        //  dejstvitel'noe okonchanie kommentarija
                        m_state = STATE_DEFAULT; //  vsjo, kommentarij zakonchilsja
                        //  tip tokena
                        m_dTokenType = TT_COMMENT;
                        //  prigotovit' otvet
                        m_finishTokenIndex = _stream.prt() - 1;
                        //  vernut'
                        return _stream.substring(m_startTokenIndex, m_finishTokenIndex);
                    }
                    //  sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
                case STATE_COMPLEX_COMMENT:
                    if (_stream.end() || _stream.isComplexCommentEnd()) {
                        //  vsjo, kommentarij zakonchilsja
                        m_state = STATE_DEFAULT;
                        //  propustit' na simvol bol'she, chem obychno
                        _stream.next();
                        _stream.next();
                        //  tip tokena
                        m_dTokenType = TT_COMMENT;
                        //  prigotovit' otvet
                        m_finishTokenIndex = _stream.prt() - 1;
                        //  vernut'
                        return _stream.substring(m_startTokenIndex, m_finishTokenIndex);
                    }
                    //  sledujuschij simvol
                    _stream.next();
                    break;
                    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
            }
        }
    }

    /**
     * sozdat' novyj interval i dobavit' ego v spisok
     */
    private void addNewInterval(int _start, int _end, int _category) {
        dIntervals.add(new IntervalInfo(new TextInterval(_start, _end + 1, 0), _category));
    }

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    private class IntervalInfo {
        public IntervalInfo(TextInterval pInterval, int pCategory) {
            dInterval = pInterval;
            dCategory = pCategory;
        }

        public TextInterval dInterval;
        public int dCategory;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
    /**
     * intervaly
     */
    private ArrayList<IntervalInfo> dIntervals = new ArrayList<IntervalInfo>();
    /**
     * iskhodnyj tekst (predyduschij razobrannyj)
     */
    private String dText = "";
    /**
     * vremennaja peremennaja ispol'zuemaja pri parsinge - khranit sostojanie
     */
    private int m_state;
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * posto sozdat' novyj razborschik
     */
    public TextMiniTokenizer() {
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * Vypolnjaet peretransljatsiju teksta v intervaly. Ispol'zuetsja invormatsija, kotoraja
     * ostalas' ot predyduschej transljatsii. V protivnom sluchae - transljatsija vypolnjaetsja zanovo.
     *
     * @return kolichestvo intervalov, kotorye ne byli izmeneny v rezul'tate etoj peretransljatsii.
     */
    public int translateText(String _text) {
        //---------------------------------------------------------------------------------------------------
        //  najti pervoe razlichie
        int firstDiff = -1;
        {
            int ln = Math.min(dText.length(), _text.length());
            for (int i = 0; i < ln; i++)
                if (dText.charAt(i) != _text.charAt(i)) {
                    firstDiff = i;
                    break;
                }
            if (firstDiff < 0) firstDiff = ln;
        }
        this.dText = _text;
        //---------------------------------------------------------------------------------------------------
        //  najti kolichestvo intervalov, kotorye ne izmenilis'
        int validIntervals = 0;
        int parseStart = -1;
        {
            //  idem s kontsa i nakhodim pervyj interval, konets kotorogo nakhoditsja levee pervogo razlichija
            for (int i = (getIntervalCount() - 1); i >= 0; i--) {
                int _end = getInterval(i).getNextCharOffset();
                if (_end < firstDiff) { // da, na etot interval uzhe nel'zja povlijat'

                    validIntervals = i;
                    //  udalit' vse intervaly posle etogo
                    while (getIntervalCount() > (validIntervals + 1)) dIntervals.remove(validIntervals + 1);
                    //  razbor teksta nuzhno nachinat' s etogo simvola
                    parseStart = _end;
                    //  dal'she idti ne nuzhno
                    break;
                }
            }
            //  esli interval ne byl najden
            if (parseStart < 0) {
                parseStart = 0;
                validIntervals = 0;
                dIntervals.clear();
            }
        }
        //---------------------------------------------------------------------------------------------------
        //  proparsirovat'
        MiniStream _stream = new MiniStream(_text, parseStart);
        m_state = STATE_DEFAULT;
        //  tsikl po tokenam
        while (true) {
            //  sledujuschij token
            String _token = this.getNextToken(_stream);
            if (_token == null) break;

            //  tip tokena
            int tt = findCategoryOfLastToken(_token);
            if (tt >= 0) {  //  dobavit' etot token s spisok interavlov

                addNewInterval(m_startTokenIndex, m_finishTokenIndex, tt);
            }
        }

        return validIntervals;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * obschee kolichestvo intervalov
     */
    public int getIntervalCount() {
        return dIntervals.size();
    }

    /**
     * interval po indeksu
     */
    public TextInterval getInterval(int _ind) {
        return dIntervals.get(_ind).dInterval;
    }

    /**
     * kategorija intervala po indeksu
     */
    public int getIntervalCategory(int _ind) {
        return dIntervals.get(_ind).dCategory;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * Poisk pervogo sleva intervala, kotoryj zakanchivaetsja posle smeschenija
     */
    public int findLeftInterval(int _beginOffset) {
        int rt = findInterval(_beginOffset);
        if (rt == -2) return -2;
        if (rt == -1) return 0;
        if ((rt < getIntervalCount()) && (_beginOffset >= getInterval(rt).getNextCharOffset())) rt++;
        return rt;
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * Poisk pervogo sprava intervala, kotoryj nachinaetsja do smeschenija
     */
    public int findRightInterval(int _endOffset) {
        return findInterval(_endOffset);
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=

    /**
     * poisk intervala po smescheniju.
     *
     * @return vozvraschaet indeks pervogo sprava intervala, nachalo kotorogo men'she, chem zadannoe smeschenie.
     */
    private int findInterval(int _offset) {
        int start = 0;
        int end = getIntervalCount();
        int current = 0;

        while (true) {
            //  negde iskat' - nuzhno vernut' -1
            if (start >= end) return -2;

            //  esli pervyj interval nachinaetsja posle smeschenija - vernut' predyduschij
            if (_offset < getInterval(start).getStartCharOffset()) return start - 1;

            //  esli poslednij interval nachinaetsja do smeschenija - vernut' poslednij
            if (_offset >= getInterval(end - 1).getStartCharOffset()) return end - 1;

            //  poluchit' sernij interval
            current = (start + end) / 2;

            if (current == start) { //  tol'ko, esli start == (end + 1), t.e. rassmatrivaemyj diapazon
                //   intervalov sostoit iz odnogo intervala

                //  vernut' etot interval
                return start;
            }

            //  inache zanovo zapustit'
            if (_offset >= getInterval(current).getStartCharOffset())
                start = current;
            else
                end = current;
        }
    }
    //-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=
}
